package com.avasthi.cli.twitter;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.Map;
import java.util.Random;
import java.util.Scanner;

import de.l3s.boilerpipe.BoilerpipeProcessingException;
import de.l3s.boilerpipe.extractors.ArticleExtractor;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import net.minidev.json.JSONValue;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;

public class TwitterJsonParserHashtagDirectory {

	public static void main(String[] args) {
		// TODO Auto-generated method stub

		if (args.length == 0) {
			System.out.println("Usage: java TwitterJsonParser twitterlog");
			System.exit(0);
		}
		File inp = new File(args[0]);
		System.out.println("Reading file :" + inp.getAbsolutePath());
		Map<Long, String> rtMap = new HashMap<Long, String>();
		try {
			FileReader fr = new FileReader(inp);
			BufferedReader reader = new BufferedReader(fr);
			JSONParser parser = new JSONParser(JSONParser.MODE_PERMISSIVE);
			JSONObject jv = null;
			Object o = null;
			while ((o = parser.parse(reader)) != null) {
				if (o instanceof java.lang.String) {
					System.out.println((String) o);
					break;
				}
				jv = (JSONObject) o;
				Long id = (Long) jv.get("id");
				JSONObject retweeted_status = (JSONObject) jv
						.get("retweeted_status");
				JSONObject entities = (JSONObject) jv.get("entities");
				if (retweeted_status != null) {

					if (entities != null) {

						{

							Long rtId = Long
									.parseLong((String) retweeted_status
											.get("id_str"));
							if (!rtMap.containsKey(rtId)) {
								rtMap.put(rtId, "");
								Random random = new Random();
								String subdir = "train";
								if (random.nextInt() % 10 == 0) {
									subdir = "test";
								}
								File f = new File("/home/vavasthi/tweetscateg/"
										+ subdir);
								if (!f.isDirectory()) {
									f.mkdirs();
								}
								JSONArray hashtags = (JSONArray) entities
										.get("hashtags");
								if (hashtags != null && hashtags.size() > 0) {
									for (JSONObject h : hashtags
											.toArray(new JSONObject[hashtags
													.size()])) {
										String hashtagCategory = h.get("text")
												.toString().toLowerCase();
										System.out.println("Found Hashtags"
												+ hashtags.size() + " "
												+ hashtagCategory);
										File catDir = new File(f,
												hashtagCategory);
										if (!catDir.isDirectory()) {
											catDir.mkdirs();
										}
										PrintWriter writer = new PrintWriter(
												new File(catDir,
														rtId.toString()),
												"UTF-8");
										writer.println((String) retweeted_status
												.get("text"));
										writer.close();
									}

								} else {

									JSONArray urls = (JSONArray) entities
											.get("urls");
									if (urls != null && urls.size() > 0) {
										for (JSONObject u : urls
												.toArray(new JSONObject[urls
														.size()])) {
											String su = u.get("expanded_url")
													.toString();
											URL url;
											try {
												url = new URL(su);

												File catDir = new File(f,
														url.getHost());
												if (!catDir.isDirectory()) {
													catDir.mkdirs();
												}
												PrintWriter writer = new PrintWriter(
														new File(catDir, rtId
																.toString()),
														"UTF-8");
												writer.println((String) retweeted_status
														.get("text"));
												writer.close();
											} catch (MalformedURLException e) {
												// TODO Auto-generated catch
												// block
												e.printStackTrace();
											}

										}
									}
								}
							}
						}
					}
				}
			}

		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
